# coding=utf-8 
import re, sys, chardet

def trim_text(path = "d:/2.txt"):
    endStr = '.。!！?？」》…节”'

    encoding = 'GB18030'
    with open(path, 'rb') as f:
        charset = chardet.detect(f.read())
        encoding = 'GB18030' if charset['encoding']=='GB2312' else charset['encoding']
        #print(encoding)

    text = ""
    with open(path, "r", encoding=encoding) as myfile:
        lines = myfile.readlines()
        
        isEnd = True
        upLineIsEnd = False

        stripEnd = re.compile(r'(\S+)\s*')
        stripBegin = re.compile(r's*(\S+)')

        for i, line in enumerate(lines):
            if not line or line.isspace():
                continue
    
            line = stripEnd.subn(r'\1', line)[0]
            isEnd = checkIsEnd(line, endStr)

            if upLineIsEnd==False:
                line = stripBegin.subn(r'\1', line)[0]

            text = text + line

            if isEnd:
                text = text + "\n\n"

            upLineIsEnd = isEnd
                
    text = clean(text)
    new_file_name = path[:path.rindex('.')] + '_' + path[path.rindex('.'):]

    open(new_file_name, "w", encoding='GB18030').write(text)

def checkIsEnd(line, endStr):
    isEnd = False
    for c in endStr:
        if line.endswith(c):
            isEnd = True
            break

    if "章" in line or "楔子" in line:
        isEnd = True
    
    return isEnd

def clean(text):
    text = re.sub(r'#\w{3};', '', text)
    text = re.sub(r'&\w{3};', '', text)
    text = text.replace("</br>", '')
    text = text.replace("amp;", '')
    text = text.replace("&hellip;", '…')
    text = text.replace("&quot;", '“')
    text = re.sub(r'#x\w{4};', '', text)
    text = re.sub(r'\d{4}-\d{1,2}-\d{1,2}', '\n', text)
    text = re.sub(r'\d{4}年\d{1,2}月\d{1,2}日', '\n', text)
    text = re.sub(r'(第\S{1,5}章)', '\n\g<1>', text)
    return text


if __name__ == '__main__':
    if len(sys.argv)<=1:
        print("No Input File!!!!!")
        trim_text()
    elif len(sys.argv)==2:
        trim_text(sys.argv[1])
